/**
 * @author Fang Lu, fanglupku@gmail.com
 * 2012-2-20
 */
package com.autotag.data.crawler;

import java.io.File;
import java.io.IOException;
import java.sql.Connection;
import java.util.Iterator;
import java.util.List;

import com.autotag.dao.ServiceDao;
import com.autotag.entity.Service;

import fanglu.util.FileDealer;
import fanglu.util.SQLDealer;

public class WSDLCacheCrawler {
	private static SimpleHTMLFetcher simpleHTMLFetcher = new SimpleHTMLFetcher();
	
	public static boolean fetchWSDLCache(Service thisService, String baseDir, String encoding, boolean isForce){
		boolean result = true;
		if (thisService.getWSDL().trim() == ""){
			return true;
		}
		String thisProviderName = ServiceDetailCrawler.extractProviderFromRef(thisService.getRefSeekda());
		String thisServiceName = ServiceDetailCrawler.extractServiceNameFromRef(thisService.getRefSeekda(), thisProviderName);
		String thisProviderLowerName = thisProviderName.toLowerCase();
		String subDirName = "/other";
		if (thisProviderLowerName.toLowerCase().charAt(0) >= 'a' && thisProviderLowerName.toLowerCase().charAt(0) <= 'z'){
			subDirName = "/" + thisProviderLowerName.toLowerCase().charAt(0);
		}
		String thisDirName = baseDir + subDirName + "/" + thisProviderName;
		File thisDir = new File(thisDirName);
		
		if (thisService.getStatus() == Service.WSDLCACHED_SERVICE && thisDir.exists() && !isForce){
			return true;
		}
		
		if (!thisDir.exists()){
			thisDir.mkdirs();
		}
		String urlString = thisService.getWSDL();
		StringBuffer sb = new StringBuffer();
		int statusCode = simpleHTMLFetcher.simpleGetPageContent(urlString, sb, encoding);
		if (statusCode == 200 && sb.indexOf("Service Temporarily Unavailable") < 0){
			String serviceFileName = thisDirName + "/" + thisServiceName.replace('?', '.').replace('/', '.').replace('-', '.') + ".wsdl.xml";
			try {
				FileDealer.saveFile(sb.toString(), serviceFileName, encoding, false);
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				result = false;
			}
		}else{
			System.out.println("error, status code:" + statusCode + "/address:" + urlString);
			result = false;
		}
		return result;
	}
	
	public static void crawlAllWSDLCache(String baseDir, String encoding, String dbURL, String user, String pwd){
		Connection conn = SQLDealer.getDbConnection(dbURL, user, pwd);
		List<Service> serviceList = ServiceDao.getAllService(conn);
		Iterator<Service> itor = serviceList.iterator();
		int totalSize = serviceList.size();
		int thisIndex = 0;
		while (itor.hasNext()){
			Service thisService = itor.next();
			int retryTime = 0;
			while (!fetchWSDLCache(thisService, baseDir, encoding, false)){
				System.out.println((thisIndex+1) + " / " + totalSize + " is failed! retry!");
				try {
					Thread.sleep(60000);
				} catch (InterruptedException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				++ retryTime;
				if (retryTime > 2){
					break;
				}
			}
			if (retryTime > 2){
				System.out.println((thisIndex+1) + " / " + totalSize + " is droped!");
			}else{
				thisService.setStatus(Service.WSDLCACHED_SERVICE);
				ServiceDao.updateService(conn, thisService);
				System.out.println((thisIndex+1) + " / " + totalSize + " is OK!");
			}
			++ thisIndex;
		}
	}
	
	public static void main(String args[]){
		crawlAllWSDLCache("d:\\seekda_data\\wsdl", "utf8",
				"jdbc:mysql://localhost:3306/autotag", "root", "ftc86524");
	}
}
